package com.push.spider.spider;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.push.common.config.TransConfig;
import com.push.common.manager.BeanManager;
import com.push.common.utils.DateUtils;
import com.push.common.utils.EncryptUtil;
import com.push.common.utils.MD5;
import com.push.common.utils.StringUtils;
import com.push.common.utils.httpclient.HttpClientResult;
import com.push.common.utils.httpclient.HttpClientUtil;
import com.push.spider.utils.UrlUtils;
import com.push.web.domain.BizArticle;
import com.push.web.domain.BizLabelSecond;
import com.push.web.domain.BizSource;
import com.push.web.service.IBizArticleAppendixService;
import com.push.web.service.IBizArticleService;
import com.push.web.service.IBizLabelSecondService;
import com.push.web.service.IBizSourceService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class ChinanewsSpider {
    private Logger logger = LoggerFactory.getLogger(this.getClass());

    private ArrayList<String> hasDownUrls = new ArrayList<>();

    private BizSource source;

    private List<BizLabelSecond> labelSeconds = new ArrayList<>();

    @Autowired
    IBizArticleService articleService = BeanManager.getBean(IBizArticleService.class);

    @Autowired
    IBizSourceService sourceService = BeanManager.getBean(IBizSourceService.class);

    @Autowired
    IBizLabelSecondService labelSecondService = BeanManager.getBean(IBizLabelSecondService.class);

    @Autowired
    IBizArticleAppendixService articleAppendixService = BeanManager.getBean(IBizArticleAppendixService.class);

    @Autowired
    private TransConfig transConfig = BeanManager.getBean(TransConfig.class);

    public int size = 0;

    public ChinanewsSpider(String sourceId){
        this.source = sourceService.selectBizSourceBySourceId(Long.valueOf(sourceId));
        this.labelSeconds = labelSecondService.selectBizLabelSecondListByParentIds(this.source.getLabelIds());
    }

    public void run(){
        HashMap params = new HashMap();
        params.put("q", source.getColumnTitle());
        params.put("ps", "10");
        params.put("start", "0");
        params.put("type", "");
        params.put("sort", "pubtime");
        params.put("time_scope", "0");
        params.put("channel", source.getCountry());
        params.put("adv", "1");
        params.put("day1", "");
        params.put("day2", "");
        params.put("field", "");
        params.put("creator", "");
        HttpClientResult clientResult = HttpClientUtil.doPost(source.getSpiderUrl(), params,true);
        Html html = new Html(clientResult.getContent());
        System.out.println("总页数："+getPages(html));
        for (int page = Integer.valueOf(source.getPageXpath()); page < getPages(html); page++){
            System.out.println("当前第"+page+"页");
            params.put("start", String.valueOf(page*10));
            clientResult = HttpClientUtil.doPost(source.getSpiderUrl(), params,true);
            html = new Html(clientResult.getContent());
            if (!StringUtils.isEmpty(source.getListXpath())) {
                List<String> listXpaths = Arrays.asList(source.getListXpath().split(";"));
                for (String listXpath:listXpaths){
                    List<Selectable> tagNodes = html.xpath(listXpath).nodes();
                    for (Selectable contentNode : tagNodes) {
                        String titleOrg = extractInfo(source.getTitleXpath(), contentNode);
                        String sourceUrl = extractInfo(source.getDetailsXpath(), contentNode);
                        String avator = extractInfo(source.getAvatorXpath(), contentNode);
                        String publishTime = extractInfo(source.getPublishTimeXpath(), contentNode);
                        String summary = extractInfo(source.getSummaryXpath(), contentNode);
                        if (StringUtils.isEmpty(sourceUrl)){
                            continue;
                        }
                        String dataKey = EncryptUtil.encrypt(sourceUrl, "md5");
                        /*
                        try {
                            Thread.sleep(1000);
                        } catch (InterruptedException e) {
                            e.printStackTrace();
                        }*/
                        HttpClientResult detailResult = HttpClientUtil.doGet(sourceUrl, true);
                        Html detailhtml = new Html(detailResult.getContent());
                        String summaryOrg = extractInfo(source.getSummaryXpath(), detailhtml);
                        String contentsOrg = extractInfo(source.getContentsXpath(), detailhtml);
                        String publisher = extractInfo(source.getPublisherXpath(), detailhtml);
                        String publishTimeNew = extractInfo(source.getPublishTimeXpath(), detailhtml);
                        String avatorNew = "";
                        List<String> contentXpathList = Arrays.asList(source.getContentsXpath().split(";"));
                        if(contentXpathList.size() > 0){
                            for (String contentXpath: contentXpathList){
                                List<Selectable> contentNodes = detailhtml.xpath(contentXpath).nodes();
                                if(contentNodes.size() > 0){
                                    /*
                                     * 单独获取图片
                                     */
                                    List<Selectable> imgNodes = contentNodes.get(0).xpath("//img/@src").nodes();
                                    if (imgNodes.size() > 0) {
                                        avatorNew = UrlUtils.buildUrl(sourceUrl, imgNodes.get(0).toString());
                                    }
                                    break;
                                }
                            }
                        }
                        BizArticle article = buildArticle(dataKey, titleOrg, sourceUrl, avator, publishTime, summary, summaryOrg, contentsOrg, publisher, publishTimeNew, avatorNew,source.getSourceId());
                        BizArticle bizArticle = articleService.selectBizArticleByDataKey(dataKey);
                        if (StringUtils.isNull(bizArticle)){
                            if (strContains(article.getTitleOrg(), labelSeconds)){
                                //自动翻译标题
                                if (StringUtils.isContainsChinese(article.getTitleOrg())){
                                    article.setTitleCn(article.getTitleOrg());
                                } else {
                                    String transWord = getTransResult(article.getTitleOrg(), "auto", "zh");
                                    JSONObject titleObj = JSONObject.parseObject(transWord);
                                    JSONArray titleArray = titleObj.getJSONArray("trans_result");
                                    if (StringUtils.isNotNull(titleArray) && titleArray.size() > 0){
                                        article.setTitleCn(titleArray.getJSONObject(0).getString("dst"));
                                    }
                                }
                                //自动翻译摘要
                                if(transConfig.isAbstractTrans() && StringUtils.isNotEmpty(article.getSummaryOrg())){
                                    String transWord = getTransResult(article.getSummaryOrg(), "auto", "zh");
                                    JSONObject titleObj = JSONObject.parseObject(transWord);
                                    JSONArray titleArray = titleObj.getJSONArray("trans_result");
                                    if (StringUtils.isNotNull(titleArray) && titleArray.size() > 0){
                                        article.setSummaryCn(titleArray.getJSONObject(0).getString("dst"));
                                    }
                                }
                                articleService.insertBizArticle(article);
                            }
                        } else {
                            article.setId(bizArticle.getId());
                            articleService.updateBizArticle(article);
                        }
                        size++;
                    }
                }
            }
        }
    }
    public static int getPages(Html html){
        String regEx="[^0-9]";
        Pattern p = Pattern.compile(regEx);
        List<Selectable> nodes = html.xpath("//div[@id='pagediv']/a").nodes();
        for (Selectable selectable: nodes) {
            if (selectable.xpath("//a/text()").toString().equalsIgnoreCase("尾页")){
                Matcher m = p.matcher(selectable.xpath("//a/@href").toString());
                return Integer.valueOf(m.replaceAll("").trim());
            }
        }
        return 0;
    }
    public String extractInfo(String xpathString, Selectable contentNode){
        if (StringUtils.isEmpty(xpathString)){
            return "";
        }
        List<String> xpathList = Arrays.asList(xpathString.split(";"));
        for (String xpath: xpathList){
            List<Selectable> nodes = contentNode.xpath(xpath).nodes();
            if(nodes.size() > 0){
                return nodes.get(0).toString();
            }
        }
        return "";
    }

    public BizArticle buildArticle(String dataKey, String titleOrg, String url,String avator, String publishTime, String summary, String summaryOrg, String contentsOrg, String publisher, String publishTimeNew, String avatorNew,long sourceId){
        BizArticle article = new BizArticle();
        article.setDataKey(dataKey);
        article.setTitleOrg(titleOrg);
        article.setUrl(url);
        if (StringUtils.isNotEmpty(avator)){
            article.setAvator(avator);
        } else {
            article.setAvator(avatorNew);
        }
        if (StringUtils.isNotEmpty(publishTime)){
            article.setPublishTime(publishTime);
            article.setPublishFormatTime(DateUtils.formatDate(DateUtils.parseDate(publishTime),publishTime));
        }else {
            article.setPublishTime(publishTimeNew);
            article.setPublishFormatTime(DateUtils.formatDate(DateUtils.parseDate(publishTimeNew),publishTimeNew));
        }
        article.setSourceId(sourceId);
        article.setCreateTime(DateUtils.getNowDate());
        if (StringUtils.isNotEmpty(summary)){
            article.setSummaryOrg(summary);
        } else {
            article.setSummaryOrg(summaryOrg);
        }
        contentsOrg.replaceAll("visibility: hidden;", "visibility: visible;");
        article.setContentsOrg(contentsOrg);
        article.setContentsCn(StringUtils.delHtmlTags(contentsOrg));
        article.setPublisher(publisher);
        return article;
    }

    public static boolean strContains(String str, List<BizLabelSecond> labels){
        if (labels == null){
            return true;
        }
        if (labels.isEmpty()){
            return true;
        }
        if(StringUtils.isEmpty(str)){
            return false;
        }
        long len = labels.parallelStream().filter(label -> str.toLowerCase(Locale.ROOT).contains(label.getLabelName().toLowerCase(Locale.ROOT))).count();
        if (len > 0){
            return true;
        }
        return false;
    }

    public String getTransResult(String query, String from, String to) {
        Map<String, String> params = buildParams(query, from, to);
        return HttpClientUtil.doGet(transConfig.getTransUrl(), params,false).getContent();
    }

    private Map<String, String> buildParams(String query, String from, String to) {
        Map<String, String> params = new HashMap<String, String>();
        params.put("q", query);
        params.put("from", from);
        params.put("to", to);
        params.put("appid", transConfig.getAppid());
        // 随机数
        String salt = String.valueOf(System.currentTimeMillis());
        params.put("salt", salt);
        // 签名
        String src = transConfig.getAppid() + query + salt + transConfig.getSecurityKey();
        params.put("sign", MD5.md5(src));
        return params;
    }

    public int getSize(){
        return size;
    }
}
